# -*- coding:utf-8 -*-

import requests
import time
from urllib import parse
from bs4 import BeautifulSoup
from selenium import webdriver
import random
from lxml import etree

#北京法院审判信息网
#https://www.bjcourt.gov.cn/cpws/index.htm

'''https://www.bjcourt.gov.cn/cpws/index.htm?st=1&q=&sxnflx=11&prompt=&dsrName=&ajmc=&ajlb=&jbfyId=1&zscq=&ay=&ah=&cwslbmc=&startCprq=&endCprq=&page=20
https://www.bjcourt.gov.cn/cpws/index.htm?st=1&q=&sxnflx=11&prompt=&dsrName=&ajmc=&ajlb=&jbfyId=1&zscq=&ay=&ah=&cwslbmc=&startCprq=&endCprq=&page=14
https://www.bjcourt.gov.cn/cpws/index.htm?st=1&q=&sxnflx=11&prompt=&dsrName=&ajmc=&ajlb=&jbfyId=1&zscq=&ay=&ah=&cwslbmc=&startCprq=2021-07-01&endCprq=2021-07-29&page=3
https://www.bjcourt.gov.cn/cpws/index.htm?st=1&q=&sxnflx=8&prompt=&dsrName=&ajmc=&ajlb=&jbfyId=1&zscq=&ay=&ah=&cwslbmc=&startCprq=2018-07-07&endCprq=2018-07-12&page=4
https://www.bjcourt.gov.cn/cpws/index.htm?st=1&q=&sxnflx=8&prompt=&dsrName=&ajmc=&ajlb=&jbfyId=16&zscq=&ay=&ah=&cwslbmc=&startCprq=2018-07-07&endCprq=2018-07-12&page=3
'''

'''sxnflx 年份 8》2018 11 》 2021
jbfyId 法院代号 

startCprq 开始日期
endCprq 结束日期
page 页码

五位图片 英文数字验证码'''
#chromesite 意义：在同一脚本中开启不同属性设置的chromedriver
#chromesite 参数：useragent-
#                ip-ip代理 默认为空 格式为str “http*://IP：port”
#                headless-可视化页面 默认为True 格式为bool
#                imge-显示图片 默认为False 格式为bool
#                linuxenv-Linux环境 默认为False 格式为bool

def chromesite(useragent = "",ip = "",headless = True,imge = False,linuxenv = False):
    chrome_options = webdriver.ChromeOptions()

    if useragent == False:
        chrome_options.add_argument(
            "user-agent:'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'")
    else:
        chrome_options.add_argument("user-agent:"+useragent)

    if headless == True:
        chrome_options.add_argument('--headless')

    if imge == False:
        prefs = {'profile.default_content_setting_values': {'images': 2}}
        chrome_options.add_experimental_option('prefs', prefs)
    if ip:
        chrome_options.add_argument('--proxy-server='+ip)

    chrome_options.add_argument('--no-sandbox')
    if linuxenv == True:
        return webdriver.Chrome("./chromedriver",options=chrome_options)

    return webdriver.Chrome(options=chrome_options)

def pa(url,urlxpath,savepath,ip,endnum,startnum=1,headless=True):
    driver = chromesite(useragent="", ip = ip, headless=headless, imge=False, linuxenv=True)

    while startnum-1!=endnum:
        print(startnum,endnum)
        oldset = set()
        #等待静态页面加载，等待时间过长则跳出这一页
        urlp=url+str(startnum)
        print(urlp)
        driver.get(urlp)
        ts = 0
        while True:
            time.sleep(1)
            driver.get(urlp)
            html = driver.page_source
            etr = etree.HTML(html)
            pagenum = etr.xpath("//span[@class='info']/em/text()")
            try:
                if int(pagenum[0]) != 0:
                    break
                else:
                    print("无文本")
                    return True
            except Exception as e:
                print("未加载")
                print(e)
                if ts<3:
                    ts +=1
                    time.sleep(3)

                    continue
                else:
                    driver.quit()
                    return False
        try:
            html = driver.page_source
            etr = etree.HTML(html)
            pageurl = etr.xpath(urlxpath)
            firsturl = "https://www.bjcourt.gov.cn/"
            for purl in pageurl:
                if purl in oldset:
                    continue
                else:
                    oldset.add(purl)
                    gurl = parse.urljoin(firsturl, purl)
                    js = 'window.open(\'' + gurl + '\');'
                    driver.execute_script(js)
                    handles = driver.window_handles
                    driver.switch_to.window(handles[1])
                    time.sleep(0.5)
                    text = []

                    soup = BeautifulSoup(driver.page_source, 'lxml')


                    for table in soup.body.select('p.MsoNormal'):
                        text.append(table.get_text())
                    with open(savepath,"a",encoding="utf-8") as w:
                        for i in text:
                            w.write(i)
                    print(gurl+' done!')
                    time.sleep(1)
                    driver.close()
                    driver.switch_to.window(handles[0])
        except Exception:
            driver.quit()
        startnum += 1
    print(url + ' done!')
    driver.quit()
    return True

#'1970-01-01','2010-12-31'
def pagenumber(jbfyid,startdate,enddate):

    driver = chromesite(useragent="", ip="", headless=True, imge=False, linuxenv=True)

    url = 'https://www.bjcourt.gov.cn/cpws/index.htm?st=1&q=&sxnflx=&prompt=&dsrName=&ajmc=&ajlb=&jbfyId={}&zscq=&ay=&ah=&cwslbmc=&startCprq={}&endCprq={}&page=1'.format(jbfyid,startdate,enddate,"{}")
    print(url)
    try:
        driver.get(url)
    except Exception as e:
        driver.quit()
        print(e)

    time.sleep(2)

    html = driver.page_source
    etr = etree.HTML(html)
    number = etr.xpath("//span[@class='info']/em/text()")
    try:
        pnumber = (int(number[0])//20)+1
    except Exception:
        driver.quit()
        return False
    driver.quit()
    return jbfyid,startdate,enddate,pnumber,number


def openIp(number):
    drivers.get("http://www.ip3366.net/?stype=1&page="+str(number))
    time.sleep(2)
    html = drivers.page_source
    etr=etree.HTML(html)
    pageurl=etr.xpath("//tbody//tr/td[1]/text() | //tbody//tr/td[2]/text() | //tbody//tr/td[4]/text()")
    print(pageurl)

    for i in range(0,len(pageurl),3):
        iplist.append(pageurl[i+2].lower()+"://"+pageurl[i]+":"+pageurl[i+1])
    drivers.quit()
    time.sleep(30)

def daili(ip):

    driver = chromesite(useragent="", ip=ip, headless=True, imge=False, linuxenv=True)

    try:
        driver.get("https://www.bjcourt.gov.cn")
        time.sleep(2)
        html = driver.page_source
        etr = etree.HTML(html)
        text = etr.xpath("/html/body/div[2]/div[3]/div/div[1]/div[2]/span[1]/p[1]/text()")
        if text[0] =="北京市高级人民法院版权所有":
            driver.quit()
            return ip
        else:
            driver.quit()
            return False
    except Exception:
        driver.quit()
        return False


openip = set()
iplist = []

drivers = chromesite(useragent="", ip="", headless=True, imge=False, linuxenv=True)



jbfyIdlist = [1,2,12,29,22,30,31,13,3,15,6,16,5,7,8,18,10,17,9,20,19,21,11,23,32]

#1384876800
datelist = []
for i in range(1434470400,1627626892,86400):# 1293811200> 2011-01-01 00:00:00
    timeArray = time.localtime(i)
    otherStyleTime = time.strftime("%Y-%m-%d", timeArray)
    datelist.append(otherStyleTime)


jbfyId = ""
ip = ""
jump = 1
errordate = -1
nowdate = 0
while True:
    try:
        for date in range(errordate+1,len(datelist),jump):
            nowdate = date
            jbfyId = ""
            print(datelist[date])
            while True:
                try:
                    pageinfo = pagenumber(jbfyId, datelist[date], datelist[date-1 + jump])
                except Exception as e:
                    print(e)
                    continue
                if pageinfo == False:
                    time.sleep(10)
                else:
                    break
            if jbfyId == "" and pageinfo[-1] == 0:
                break
            if pageinfo[-1] == 0:
                continue

            while jump > 1 and pageinfo[-2] > 10:
                print(jump,pageinfo[-2])
                jump = int(jump/2)
                try:
                    pageinfo = pagenumber(jbfyId, datelist[date], datelist[date-1 + jump])
                except Exception as e:
                    print(e)
                    continue
                if jump == 1:
                    break

            if jump == 1 and pageinfo[-2] > 10:
                for jbfyId in jbfyIdlist:
                    while True:
                        try:
                            pageinfo = pagenumber(jbfyId, datelist[date], datelist[date - 1 + jump])
                            break
                        except Exception as e:
                            print(e)
                            continue
                    if pageinfo == False:
                        continue
                    numpage = pageinfo[-2]
                    print(pageinfo[-1])
                    if pageinfo[-2] > 10:
                        numpage = 10
                        with open("uprange.txt","a",encoding="utf-8") as w:
                            w.write(str(jbfyId))
                            w.write("\t")
                            w.write(datelist[date])
                            w.write("\t")
                            w.write(str(pageinfo[-2]))
                            w.write("\n")
                    print(jbfyId)
                    state = pa(
                        'https://www.bjcourt.gov.cn/cpws/index.htm?st=1&q=&sxnflx=&prompt=&dsrName=&ajmc=&ajlb=&jbfyId={}&zscq=&ay=&ah=&cwslbmc=&startCprq={}&endCprq={}&page='.format(
                            jbfyId, pageinfo[1], pageinfo[2]),
                        "/html/body/div[1]/div[3]/div[3]/div/div[2]/ul//li/a/@href",
                        "save.txt",
                        ip,
                        numpage,
                    )
                    if jbfyId != 32:
                        continue
                    if state == False:
                        time.sleep(100)
                        continue
                    if state == True:
                        break
                        '''while True:
                            if len(openip) == 0:
                                for i in range(1, 11):
                                    openip.add(i)
                            if len(iplist) == 0:
                                openIp(openip.pop())
    
                            for i in iplist:
                                if daili(i):
                                    ip = i
                                    break
                                else:
                                    continue
                            break
                        time.sleep(3)'''
            else:
                jbfyId = ""
                print(jbfyId)
                while True:
                    state = pa(
                        'https://www.bjcourt.gov.cn/cpws/index.htm?st=1&q=&sxnflx=&prompt=&dsrName=&ajmc=&ajlb=&jbfyId={}&zscq=&ay=&ah=&cwslbmc=&startCprq={}&endCprq={}&page='.format(pageinfo[0], pageinfo[1], pageinfo[2]),
                        "/html/body/div[1]/div[3]/div[3]/div/div[2]/ul//li/a/@href",
                        "save.txt",
                        ip,
                        pageinfo[-2],
                    )
                    if state == False:
                        time.sleep(100)
                        continue
                    if state == True:

                        break
    except Exception as e:
        errordate = nowdate
        with open("errorlog.txt","a",encoding="utf-8")as w:
            w.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
            w.write("\t")
            w.write(str(e))
            w.write("\t")
            w.write("jbfyId=")
            w.write(str(jbfyId))
            w.write("\t")
            w.write("date=")
            w.write(str(datelist[date]))
            w.write("\n")

'''if len(openip) == 0:
                        for i in range(1, 11):
                            openip.add(i)
                    if len(iplist) == 0:
                        openIp(openip.pop())

                    for i in iplist:
                        if daili(i):
                            ip = i
                            break
                        else:
                            continue
                    break
                time.sleep(3)'''


''' if int(pagenumber(jbfyId,datelist[date],datelist[date+jump])[0]) > 400 and jump > 1:
            jump = int(jump/2)
            break

        if int(pagenumber(jbfyId,datelist[date],datelist[date+jump])[0]) > 400 and jump == 1:
            for jbfyId in jbfyIdlist:
                pa(
                    pagenumber(jbfyId, datelist[date], datelist[date + jump])[0],
                    "/html/body/div[1]/div[3]/div[3]/div/div[2]/ul//li/a/@href",
                    "save.txt",
                    ip,
                    pagenumber(jbfyId, datelist[date], datelist[date + jump])[1]
                )
'''